{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copyright 2019 Google LLC\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "#     https://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modern CNN Architectures - Lab 1\n",
    "\n",
    "### Overview\n",
    "\n",
    "In this lab, you will attempt to find an improvement on a mini-VGG for CIFAR-10.\n",
    "\n",
    "If one tried training a CIFAR-10 with a full VGG16 or VGG19, it will not learn. This is because the input size of CIFAR-10 is very small (32x32), so by the time you get to the bottleneck layer (last convolution before flattening), the feature maps will be only 1x1 pixels -- and thus have no spatial information. Conventional practice is 4x4 is ideal, but 3x3 generally works.\n",
    "\n",
    "Below is a composable \"class\" based version for building VGG networks. Spend a few moments looking at the structure and get familiar."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " <a target=\"_blank\" href=\"https://colab.research.google.com/github/GoogleCloudPlatform/keras-idiomatic-programmer/blob/master/workshops/Modern_CNN/Modern%20CNN%20-%20lab%201.ipynb\">\n",
    " <img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# VGG (16 and 19 & Composable) (2014)\n",
    "# Paper: https://arxiv.org/pdf/1409.1556.pdf\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import Input, Model\n",
    "from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense\n",
    "\n",
    "class VGG(object):\n",
    "    \"\"\" VGG (composable)\n",
    "    \"\"\"\n",
    "    # Meta-parameter: list of groups: number of layers and filter size\n",
    "    groups = { 16 : [ (1, 64), (2, 128), (3, 256), (3, 512), (3, 512) ],\t# VGG16\n",
    "               19 : [ (1, 64), (2, 128), (4, 256), (4, 256), (4, 256) ] }\t# VGG19\n",
    "\n",
    "    init_weights='glorot_uniform'\n",
    "    _model = None\n",
    " \n",
    "\n",
    "    def __init__(self, n_layers, input_shape=(224, 224, 3), n_classes=1000):\n",
    "        \"\"\" Construct a VGG model\n",
    "            n_layers    : number of layers (16 or 19)\n",
    "            input_shape : input shape to the model\n",
    "            n_classes:  : number of output classes\n",
    "        \"\"\"\n",
    "        if n_layers not in [16, 19]:\n",
    "            raise Exception(\"VGG: Invalid value for n_layers\")\n",
    "            \n",
    "        # The input vector \n",
    "        inputs = Input( input_shape )\n",
    "\n",
    "        # The stem group\n",
    "        x = self.stem(inputs)\n",
    "\n",
    "        # The learner\n",
    "        x = self.learner(x, self.groups[n_layers])\n",
    "\n",
    "        # The classifier\n",
    "        outputs = self.classifier(x, n_classes)\n",
    "\n",
    "        # Instantiate the Model\n",
    "        self._model = Model(inputs, outputs)\n",
    "\n",
    "    @property\n",
    "    def model(self):\n",
    "        return self._model\n",
    "\n",
    "    @model.setter\n",
    "    def model(self, _model):\n",
    "        self._model = _model\n",
    "    \n",
    "    def stem(self, inputs):\n",
    "        \"\"\" Construct the Stem Convolutional Group\n",
    "            inputs : the input vector\n",
    "        \"\"\"\n",
    "        x = Conv2D(64, (3, 3), strides=(1, 1), padding=\"same\", activation=\"relu\",\n",
    "                   kernel_initializer=self.init_weights)(inputs)\n",
    "        return x\n",
    "    \n",
    "    def learner(self, x, blocks):\n",
    "        \"\"\" Construct the (Feature) Learner\n",
    "            x        : input to the learner\n",
    "            blocks   : list of groups: filter size and number of conv layers\n",
    "        \"\"\" \n",
    "        # The convolutional groups\n",
    "        for n_layers, n_filters in blocks:\n",
    "            x = self.group(x, n_layers, n_filters)\n",
    "        return x\n",
    "\n",
    "    @staticmethod\n",
    "    def group(x, n_layers, n_filters, init_weights=None):\n",
    "        \"\"\" Construct a Convolutional Group\n",
    "            x        : input to the group\n",
    "            n_layers : number of convolutional layers\n",
    "            n_filters: number of filters\n",
    "        \"\"\"\n",
    "        if init_weights is None:\n",
    "            init_weights = VGG.init_weights\n",
    "        # Block of convolutional layers\n",
    "        for n in range(n_layers):\n",
    "            x = Conv2D(n_filters, (3, 3), strides=(1, 1), padding=\"same\", activation=\"relu\",\n",
    "                       kernel_initializer=init_weights)(x)\n",
    "        \n",
    "        # Max pooling at the end of the block\n",
    "        x = MaxPooling2D(2, strides=(2, 2))(x)\n",
    "        return x\n",
    "    \n",
    "    def classifier(self, x, n_classes):\n",
    "        \"\"\" Construct the Classifier\n",
    "            x         : input to the classifier\n",
    "            n_classes : number of output classes\n",
    "        \"\"\"\n",
    "        # Flatten the feature maps\n",
    "        x = Flatten()(x)\n",
    "    \n",
    "        # Two fully connected dense layers\n",
    "        x = Dense(4096, activation='relu', kernel_initializer=self.init_weights)(x)\n",
    "        x = Dense(4096, activation='relu', kernel_initializer=self.init_weights)(x)\n",
    "\n",
    "        # Output layer for classification \n",
    "        x = Dense(n_classes, activation='softmax', kernel_initializer=self.init_weights)(x)\n",
    "        return x\n",
    "\n",
    "# Example of constructing a VGG 16\n",
    "# vgg = VGG(16)\n",
    "# model = vgg.model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Starting mini-VGG\n",
    "\n",
    "Below is a mini-VGG I wrote for CIFAR-10. Notice how at the bottleneck layer the feature maps are 3 x 3 (max pooling). \n",
    "\n",
    "#### Model Summary\n",
    "\n",
    "```\n",
    "    Layer (type)                 Output Shape              Param #   \n",
    "=================================================================\n",
    "input_4 (InputLayer)         [(None, 32, 32, 3)]       0         \n",
    "_________________________________________________________________\n",
    "conv2d_26 (Conv2D)           (None, 28, 28, 32)        2432      \n",
    "_________________________________________________________________\n",
    "conv2d_27 (Conv2D)           (None, 28, 28, 64)        18496     \n",
    "_________________________________________________________________\n",
    "max_pooling2d_11 (MaxPooling (None, 14, 14, 64)        0         \n",
    "_________________________________________________________________\n",
    "conv2d_28 (Conv2D)           (None, 14, 14, 128)       73856     \n",
    "_________________________________________________________________\n",
    "conv2d_29 (Conv2D)           (None, 14, 14, 128)       147584    \n",
    "_________________________________________________________________\n",
    "max_pooling2d_12 (MaxPooling (None, 7, 7, 128)         0         \n",
    "_________________________________________________________________\n",
    "flatten_3 (Flatten)          (None, 6272)              0         \n",
    "_________________________________________________________________\n",
    "dense_5 (Dense)              (None, 10)                62730     \n",
    "=================================================================\n",
    "Total params: 305,098\n",
    "Trainable params: 305,098\n",
    "Non-trainable params: 0\n",
    "```\n",
    "\n",
    "#### Training\n",
    "\n",
    "Below is the results for training for 10 epochs. Notice how the validation accuracy plateaus out around 74%. Perhaps it is overfitting to the data. \n",
    "\n",
    "```\n",
    " validate on 5000 samples\n",
    "Epoch 1/10\n",
    "45000/45000 [==============================] - 73s 2ms/sample - loss: 1.4581 - acc: 0.4728 - val_loss: 1.1139 - val_acc: 0.6030\n",
    "Epoch 2/10\n",
    "45000/45000 [==============================] - 77s 2ms/sample - loss: 0.9879 - acc: 0.6559 - val_loss: 0.8826 - val_acc: 0.6948\n",
    "Epoch 3/10\n",
    "45000/45000 [==============================] - 81s 2ms/sample - loss: 0.7916 - acc: 0.7264 - val_loss: 0.8561 - val_acc: 0.7152\n",
    "Epoch 4/10\n",
    "45000/45000 [==============================] - 81s 2ms/sample - loss: 0.6645 - acc: 0.7689 - val_loss: 0.7758 - val_acc: 0.7362\n",
    "Epoch 5/10\n",
    "45000/45000 [==============================] - 82s 2ms/sample - loss: 0.5571 - acc: 0.8058 - val_loss: 0.7687 - val_acc: 0.7568\n",
    "Epoch 6/10\n",
    "45000/45000 [==============================] - 82s 2ms/sample - loss: 0.4691 - acc: 0.8349 - val_loss: 0.7511 - val_acc: 0.7558\n",
    "Epoch 7/10\n",
    "45000/45000 [==============================] - 82s 2ms/sample - loss: 0.3811 - acc: 0.8669 - val_loss: 0.8617 - val_acc: 0.7520\n",
    "Epoch 8/10\n",
    "45000/45000 [==============================] - 82s 2ms/sample - loss: 0.3132 - acc: 0.8897 - val_loss: 0.9241 - val_acc: 0.7468\n",
    "Epoch 9/10\n",
    "45000/45000 [==============================] - 81s 2ms/sample - loss: 0.2583 - acc: 0.9076 - val_loss: 1.0457 - val_acc: 0.7438\n",
    "Epoch 10/10\n",
    "45000/45000 [==============================] - 83s 2ms/sample - loss: 0.2174 - acc: 0.9221 - val_loss: 1.1191 - val_acc: 0.7428\n",
    "```\n",
    "\n",
    "#### Try to Improve\n",
    "\n",
    "How could we improve this? \n",
    "\n",
    "1. Perhaps adding regularization (dropout)?\n",
    "\n",
    "2. Perhaps adding batch normalization between the two VGG groups?\n",
    "\n",
    "3. Perhaps adding a squeeze (dimensionality reduction) group --model has less parameters to train?\n",
    "\n",
    "4. Perhaps adding another VGG group with doubling of filters (dimensionality expansion) --model has more parameters to train.\n",
    "\n",
    "If this is a classroom, we will split into 4 teams and each team will use a different approach.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import Input, Model\n",
    "from tensorflow.keras.layers import Conv2D, Flatten, Dense, GlobalAveragePooling2D, Reshape\n",
    "\n",
    "# Stem\n",
    "inputs = Input((32, 32, 3))\n",
    "x = Conv2D(32, (3, 3), strides=1, padding='valid', activation='relu')(inputs)\n",
    "\n",
    "# Learner\n",
    "# VGG group: 1 conv layer, 64 filters\n",
    "# VGG group: 2 conv layers, 128 filters\n",
    "x = VGG.group(x, 1, 64)\n",
    "x = VGG.group(x, 2, 128)\n",
    "\n",
    "\n",
    "# Classifier\n",
    "x = Flatten()(x)\n",
    "outputs = Dense(10, activation='softmax')(x)\n",
    "model = Model(inputs, outputs)\n",
    "model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['acc'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the model on CIFAR-10\n",
    "from tensorflow.keras.datasets import cifar10\n",
    "import numpy as np\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "x_train = (x_train / 255.0).astype(np.float32)\n",
    "x_test  = (x_test  / 255.0).astype(np.float32)\n",
    "model.fit(x_train, y_train, epochs=10, batch_size=32, validation_split=0.1, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
